﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Text;

using Microsoft.Scripting;
using Microsoft.Scripting.Runtime;
using Microsoft.Scripting.Utils;

using Scala.Runtime;

#if CLR2
using Microsoft.Scripting.Math;
#else
using System.Numerics;
#endif

namespace Scala.Compiler
{
    public sealed class Tokenizer : TokenizerService
    {
        private const char LF = '\u000A';
        private const char FF = '\u000C';
        private const char CR = '\u000D';
        private const char SU = '\u001A';

        private State _state;
        private readonly bool _verbatim;
        internal bool _dontImpyDedent;
        private bool _disableLineFeedLineSeparator;
        private SourceUnit _sourceUnit;
        private ErrorSink _errors;
        private bool _endContinues, _printFunction, _unicodeLiterals;
        private List<int> _newLineLocations;
        private SourceLocation _initialLocation;
        private TextReader _reader;
        private char[] _buffer;
        private bool _multiEolns;
        private int _position, _end, _tokenEnd, _start, _tokenStartIndex, _tokenEndIndex;
        private bool _bufferResized;
        private int _openComments = 0;

        private const int EOF = -1;
        private const int MAXIDENT = 80;
        private const int DefaultBufferCapacity = 1024;

        private Dictionary<object, NameToken> _names;
        private static object _currentName = new object();

        private int _Offset;

        public Tokenizer()
        {
            _errors = ErrorSink.Null;
            _verbatim = true;
            _state = new State(null);
            _names = new Dictionary<object, NameToken>(128, new TokenEqualityComparer(this));
        }

        public Tokenizer(ErrorSink errorSink)
        {
            ContractUtils.RequiresNotNull(errorSink, "errorSink");

            _errors = errorSink;
            _state = new State(null);
            _names = new Dictionary<object, NameToken>(128, new TokenEqualityComparer(this));
        }
        
        public override bool IsRestartable
        {
            get { return true; }
        }

        public override object CurrentState
        {
            get { return _state; }
        }

        public override SourceLocation CurrentPosition
        {
            get { return IndexToLocation(CurrentIndex); }
        }

        public SourceLocation IndexToLocation(int index)
        {
            int match = _newLineLocations.BinarySearch(index);
            if (match < 0)
            {
                if (match == -1)
                {
                    return new SourceLocation(index + _initialLocation.Index, _initialLocation.Line, checked(index + _initialLocation.Column));
                }
                match = ~match - 1;
            }
            return new SourceLocation(index + _initialLocation.Index, _sourceUnit.MapLine(match + 2) + _initialLocation.Line - 1, index - _newLineLocations[match] + _initialLocation.Column);
        }

        public SourceUnit SourceUnit
        {
            get { return _sourceUnit; }
        }

        public override ErrorSink ErrorSink
        {
            get
            {
                return _errors;
            }
            set
            {
                ContractUtils.RequiresNotNull(value, "value");
                _errors = value;
            }
        }

        public bool IsEndOfFile
        {
            get { return Peek() == EOF; }
        }

        public IndexSpan TokenSpan
        {
            get { return new IndexSpan(_tokenStartIndex, _tokenEndIndex - _tokenStartIndex); }
        }

        public void Initialize(SourceUnit sourceUnit)
        {
            ContractUtils.RequiresNotNull(sourceUnit, "sourceUnit");
            Initialize(null, sourceUnit.GetReader(), sourceUnit, SourceLocation.MinValue, DefaultBufferCapacity);
        }

        public override void Initialize(object state, System.IO.TextReader sourceReader, Microsoft.Scripting.SourceUnit sourceUnit, Microsoft.Scripting.SourceLocation initialLocation)
        {
            Initialize(state, sourceReader, sourceUnit, initialLocation, DefaultBufferCapacity);
        }

        public void Initialize(object state, TextReader reader, SourceUnit sourceUnit, SourceLocation initialLocation, int bufferCapacity)
        {
            ContractUtils.RequiresNotNull(reader, "reader");
            Tokens.Addkeywords();

            if (state != null)
            {
                if (!(state is State)) throw new Exception("Bad State");
                _state = new State((State)state);
            }
            else
            {
                _state = new State(null);
            }

            _sourceUnit = sourceUnit;
            _disableLineFeedLineSeparator = reader is NoLineFeedSourceContentProvider.Reader;

            _reader = reader;

            if (_buffer == null || _buffer.Length < bufferCapacity)
            {
                _buffer = new char[bufferCapacity];
            }

            _newLineLocations = new List<int>();
            _tokenEnd = -1;
            _multiEolns = !_disableLineFeedLineSeparator;
            _initialLocation = initialLocation;

            _tokenEndIndex = -1;
            _tokenStartIndex = 0;

            _start = _end = 0;
            _position = 0;

            DumpBeginningOfUnit();
        }

        public override TokenInfo ReadToken()
        {
            if (_buffer == null)
            {
                throw new InvalidOperationException("Uninitialized");
            }

            TokenInfo result = new TokenInfo();
            Token token = GetNextToken();
            result.SourceSpan = new SourceSpan(IndexToLocation(TokenSpan.Start), IndexToLocation(TokenSpan.End));

            switch (token.Kind)
            {
                case TokenKind.EOF:
                    result.Category = TokenCategory.EndOfStream;
                    break;

                case TokenKind.COMMENT:
                    result.Category = TokenCategory.Comment;
                    break;

                case TokenKind.Name:
                    result.Category = TokenCategory.Identifier;
                    break;

                case TokenKind.Constant:
                    result.Category = (token.Value is string) ? TokenCategory.StringLiteral : TokenCategory.NumericLiteral;
                    break;

                case TokenKind.LPAREN:
                    result.Category = TokenCategory.Grouping;
                    result.Trigger = TokenTriggers.MatchBraces | TokenTriggers.ParameterStart;
                    break;

                case TokenKind.RPAREN:
                    result.Category = TokenCategory.Grouping;
                    result.Trigger = TokenTriggers.MatchBraces | TokenTriggers.ParameterEnd;
                    break;

                case TokenKind.LBRACKET:
                case TokenKind.LBRACE:
                case TokenKind.RBRACKET:
                case TokenKind.RBRACE:
                    result.Category = TokenCategory.Grouping;
                    result.Trigger = TokenTriggers.MatchBraces;
                    break;

                case TokenKind.COLON:
                    result.Category = TokenCategory.Delimiter;
                    break;

                case TokenKind.SEMI:
                    result.Category = TokenCategory.Delimiter;
                    break;

                case TokenKind.COMMA:
                    result.Category = TokenCategory.Delimiter;
                    result.Trigger = TokenTriggers.ParameterNext;
                    break;

                case TokenKind.DOT:
                    result.Category = TokenCategory.Operator;
                    result.Trigger = TokenTriggers.MemberSelect;
                    break;

                case TokenKind.NEWLINE:
                    result.Category = TokenCategory.WhiteSpace;
                    break;

                default:
                    result.Category = TokenCategory.Operator;
                    break;
            }
            return result;
        }

        internal bool TryGetTokenString(int len, out string tokenString)
        {
            if (len != TokenLength)
            {
                tokenString = null;
                return false;
            }
            tokenString = GetTokenString();
            return true;
        }

        internal bool PrintFunction
        {
            get { return _printFunction; }
            set { _printFunction = value; }
        }

        internal bool UnicodeLiterals
        {
            get { return _unicodeLiterals; }
            set { _unicodeLiterals = value; }
        }

        public Token GetNextToken()
        {
            Token result;
            result = Next();
            DumpToken(result);
            return result;
        }

        private Token Next()
        {
            bool at_beginning = AtBeginning;

            if (_state.IncompleteString != null && Peek() != EOF)
            {
                IncompleteString prev = _state.IncompleteString;
                _state.IncompleteString = null;
                return null;
            }

            DiscardToken();

            int ch = NextChar();

            while (true)
            {
                switch (ch)
                {
                    case EOF:
                        return ReadEOF();
                    case ' ':
                    case '\t':
                        ch = SkipWhiteSpace(at_beginning);
                        break;
                    case FF:
                        ch = NextChar();
                        break;
                    case '`':
                        getBackquotedIdent();
                        break;

                    case '/':
                        if (NextChar('*'))
                        {
                            ch = skipToCommentEnd();
                            break;
                        }
                        if(NextChar('/')){
                            ch = SkipSingleLineComment();
                            break;
                        }
                        return getOperatorRest();

                    case '\'':
                        ch = NextChar();
                        if (IsNameStart(ch)) return charLitOr(ReadName);
                        //else if (isOperatorPart(ch) && ch != '\\') charLitOr(getOperatorPart);
                        else
                        {
                            ch = NextChar();
                            if (ch == '\'')
                            {
                                var token = new TermNameToken(GetTokenString(), TokenKind.CHARLIT);
                                MarkTokenEnd();
                            }
                            else
                            {
                                ReportSyntaxError(new IndexSpan(_tokenStartIndex, _position - _tokenStartIndex), "unclosed character literal", ErrorCodes.SyntaxError);
                            }
                        }
                        break;
                    case '\"':
                        return ReadString((char)ch, false, false, false);

                    case '.':
                        ch = Peek();
                        if (ch >= '0' && ch <= '9')
                            return ReadFraction();

                        MarkTokenEnd();
                        return Tokens.DotToken;

                    case ';':
                        MarkTokenEnd();
                        return Tokens.SemiToken;
                    case ',':
                        MarkTokenEnd();
                        return Tokens.CommaToken;
                    case '(':
                        _state.ParenLevel++;
                        MarkTokenEnd();
                        return Tokens.LparenToken;
                    case ')':
                        _state.ParenLevel--;
                        MarkTokenEnd();
                        return Tokens.RparenToken;
                    case '{':
                        _state.BraceLevel++;
                        MarkTokenEnd();
                        return Tokens.LbraceToken;
                    case '}':
                        _state.BraceLevel--;
                        MarkTokenEnd();
                        return Tokens.RbraceToken;
                    case '[':
                        _state.BracketLevel++;
                        MarkTokenEnd();
                        return Tokens.LbracketToken;
                    case ']':
                        _state.BracketLevel--;
                        MarkTokenEnd();
                        return Tokens.RbracketToken;
                    case SU:
                        if (_position >= _end)
                        {
                            MarkTokenEnd();
                            return Tokens.EndOfFileToken;
                        }
                        return BadChar(ch);
                    case '<':
                        return getOperatorRest();

                    case '~':
                    case '!':
                    case '@':
                    case '#':
                    case '%':
                    case '^':
                    case '*':
                    case '+':
                    case '-':
                    case '>':
                    case '?':
                    case ':':
                    case '=':
                    case '&':
                    case '|':
                    case '\\':
                        return getOperatorRest();

                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        return ReadNumber(ch);


                    default:
                        if (ReadEolnOpt(ch) > 0)
                        {
                            _newLineLocations.Add(CurrentIndex);
                            DiscardToken();
                            ch = NextChar();
                            break;
                        }
                        if (ch == '\u21D2') { return Tokens.ArrowToken; }
                        if (ch == '\u2190') { return Tokens.LarrowToken; }
                        else if (IsNameStart(ch))
                        {
                            return ReadName();
                        }
                        Token res = getOperatorRest();
                        if (res != null)
                        {
                            return res;
                        }

                        if (IsNameStart(ch)) return ReadName();

                        MarkTokenEnd();
                        return BadChar(ch);
                }
            }
        }

        private Token charLitOr(Func<Token> ReadName)
        {
            int ch = NextChar();
            if (ch == '\'')
            {
                return new TermNameToken(GetTokenString(), TokenKind.CHARLIT);
            }
            else
            {
                var token = ReadName();
                return new TermNameToken(token.Image, TokenKind.SYMBOLLIT);
            }
        }

        private void getBackquotedIdent()
        {
            throw new NotImplementedException();
        }

        private int skipToCommentEnd()
        {
            int ch = NextChar();

            _openComments += 1;
            var isDocComment = (ch == '*');
            while (true)
            {
                switch (ch)
                {
                    case '*':
                        if (NextChar('/'))
                        {
                            return NextChar();
                        }
                        ch = NextChar();
                        break;
                    default:
                        ch = NextChar();
                        break;
                }
            }
        }

        private Token getOperatorRest()
        {
            BufferBack();
            int ch = NextChar();

            while (true)
            {
                switch (ch)
                {
                    case '~':
                    case '!':
                    case '@':
                    case '#':
                    case '%':
                    case '^':
                    case '*':
                    case '+':
                    case '-':
                    case '<':
                    case '>':
                    case '?':
                    case ':':
                    case '=':
                    case '&':
                    case '|':
                    case '\\':
                        ch = NextChar();
                        break;
                    case '/':
                        ch = NextChar();
                        break;
                    default:
                        Token t;
                        BufferBack();
                        MarkTokenEnd();
                        if (Tokens.AllKeywords.TryGetValue(GetTokenString(), out t)) return t;
                        t = new TermNameToken(GetTokenString(), TokenKind.Name);
                        return t;
                }
            }
        }

        private Token CheckXML()
        {
            var last = (_position >= 2) ? _buffer[_position - 2] : ' ';
            int ch = NextChar();
            switch (ch)
            {
                case ' ':
                case '\t':
                case '\n':
                case '{':
                case '(':
                case '>':
                    if (IsNameStart(ch) || ch == '!' || ch == '?')
                    {
                        ReportSyntaxError(new IndexSpan(_tokenStartIndex, _position - _tokenStartIndex), "XML not yet supported", ErrorCodes.SyntaxError);
                    }
                    return null;
                default:
                    return getOperatorRest();
            }
        }
        private int SkipWhiteSpace(bool atBeginning)
        {
            int ch;
            do { ch = NextChar(); } while (ch == ' ' || ch == '\t');
            BufferBack();

            //if(atBeginning && ch != '#' && )
            DiscardToken();
            SeekRelative(+1);
            return ch;
        }

        private int SkipSingleLineComment()
        {
            int ch = ReadLine();
            MarkTokenEnd();

            DiscardToken();
            SeekRelative(+1);

            return ch;
        }

        private Token ReadSingleLineComment()
        {
            ReadLine();
            MarkTokenEnd();

            return new CommentToken(GetTokenString());
        }

        private Token ReadNameOrUnicodeString()
        {
            return null;
        }

        private Token ReadNameOrBytes()
        {
            return null;
        }

        private Token ReadNameOrRawString()
        {
            return null;
        }

        private Token ReadEOF()
        {
            MarkTokenEnd();
            return Tokens.EndOfFileToken;
        }

        private static ErrorToken BadChar(int ch)
        {
            return new ErrorToken(StringUtils.AddSlashes(((char)ch).ToString()));
        }

        private static bool IsNameStart(int ch)
        {
            return char.IsLetter((char)ch) || ch == '_';
        }

        private static bool IsNamePart(int ch)
        {
            return Char.IsLetterOrDigit((char)ch) || ch == '_';
        }

        #region Reading Strings

        private Token ReadString(char quote, bool isRaw, bool isUni, bool isBytes)
        {
            int sadd = 0;
            bool isTriple = false;

            if (NextChar(quote))
            {
                if (NextChar(quote))
                {
                    isTriple = true; sadd += 3;
                }
                else
                {
                    BufferBack();
                    sadd++;
                }
            }
            else
            {
                sadd++;
            }

            if (isRaw) sadd++;
            if (isUni) sadd++;
            if (isBytes) sadd++;

            return ContinueString(quote, isRaw, isUni, isBytes, isTriple, sadd);
        }

        private Token ContinueString(char quote, bool isRaw, bool isUnicode, bool isBytes, bool isTriple, int startAdd)
        {
            int end_add = 0;
            int eol_size = 0;

            for (; ; )
            {
                int ch = NextChar();

                if (ch == EOF)
                {
                    BufferBack();

                    if (isTriple)
                    {
                        // CPython reports the multi-line string error as if it is a single line
                        // ending at the last char in the file.

                        MarkTokenEnd();

                        var errorEnd = new SourceLocation(BufferTokenEnd.Index - 1, BufferTokenEnd.Line, IndexToLocation(_tokenStartIndex).Column + _tokenEndIndex - _tokenStartIndex - 1);
                        ReportSyntaxError(new SourceSpan(errorEnd, errorEnd), Resources.EofInTripleQuotedString, ErrorCodes.SyntaxError | ErrorCodes.IncompleteToken);
                    }
                    else
                    {
                        MarkTokenEnd();
                    }

                    UnexpectedEndOfString(isTriple, isTriple);
                    string incompleteContents = GetTokenSubstring(startAdd, TokenLength - startAdd - end_add);

                    _state.IncompleteString = new IncompleteString(quote == '\'', isRaw, isUnicode, isTriple);
                    return new IncompleteStringErrorToken(Resources.EofInString, incompleteContents);
                }
                else if (ch == quote)
                {

                    if (isTriple)
                    {
                        if (NextChar(quote) && NextChar(quote))
                        {
                            end_add += 3;
                            break;
                        }
                    }
                    else
                    {
                        end_add++;
                        break;
                    }

                }
                else if (ch == '\\')
                {

                    ch = NextChar();

                    if (ch == EOF)
                    {
                        BufferBack();

                        MarkTokenEnd();
                        UnexpectedEndOfString(isTriple, isTriple);

                        string incompleteContents = GetTokenSubstring(startAdd, TokenLength - startAdd - end_add - 1);

                        _state.IncompleteString = new IncompleteString(quote == '\'', isRaw, isUnicode, isTriple);

                        return new IncompleteStringErrorToken(Resources.EofInString, incompleteContents);
                    }
                    else if ((eol_size = ReadEolnOpt(ch)) > 0)
                    {

                        // skip \<eoln> unless followed by EOF:
                        if (Peek() == EOF)
                        {

                            // backup over the eoln:
                            SeekRelative(-eol_size);
                            MarkTokenEnd();

                            // incomplete string in the form "abc\

                            string incompleteContents = GetTokenSubstring(startAdd, TokenLength - startAdd - end_add - 1);

                            UnexpectedEndOfString(isTriple, true);
                            return new IncompleteStringErrorToken(Resources.EofInString, incompleteContents);
                        }

                    }
                    else if (ch != quote && ch != '\\')
                    {
                        BufferBack();
                    }

                }
                else if ((eol_size = ReadEolnOpt(ch)) > 0)
                {
                    _newLineLocations.Add(CurrentIndex);
                    if (!isTriple)
                    {
                        // backup over the eoln:
                        SeekRelative(-eol_size);

                        MarkTokenEnd();
                        UnexpectedEndOfString(isTriple, false);

                        string incompleteContents = GetTokenSubstring(startAdd, TokenLength - startAdd - end_add);

                        return new IncompleteStringErrorToken((quote == '"') ? Resources.NewLineInDoubleQuotedString : Resources.NewLineInSingleQuotedString, incompleteContents);
                    }
                }
            }

            MarkTokenEnd();

            return MakeStringToken(quote, isRaw, isUnicode, isBytes, isTriple, _start + startAdd, TokenLength - startAdd - end_add);
        }

        private Token MakeStringToken(char quote, bool isRaw, bool isUnicode, bool isBytes, bool isTriple, int start, int length)
        {
            if (!isBytes)
            {
                string contents = LiteralParser.ParseString(_buffer, start, length, isRaw, isUnicode || UnicodeLiterals, !_disableLineFeedLineSeparator);
                if (isUnicode)
                {
                    return new UnicodeStringToken(contents);
                }
                return new ConstantValueToken(contents);
            }
            else
            {
                List<byte> data = LiteralParser.ParseBytes(_buffer, start, length, isRaw, !_disableLineFeedLineSeparator);
                if (data.Count == 0)
                {
                    return null;
                }

                return null;
            }
        }

        private void UnexpectedEndOfString(bool isTriple, bool isIncomplete)
        {
            string message = isTriple ? Resources.EofInTripleQuotedString : Resources.EolInSingleQuotedString;
            int error = isIncomplete ? ErrorCodes.SyntaxError | ErrorCodes.IncompleteToken : ErrorCodes.SyntaxError;

            ReportSyntaxError(BufferTokenSpan, "Check", ErrorCodes.SyntaxError);
        }

        #endregion

        #region Reading Number

        private Token ReadNumber(int start)
        {
            int b = 10;
            if (start == '0')
            {
                if (NextChar('x') || NextChar('X'))
                {
                    return ReadHexNumber();
                }
                else if (NextChar('o') || NextChar('O'))
                {
                    return ReadOctalNumber();
                }
                b = 8;
            }
            while (true)
            {
                int ch = NextChar();

                switch (ch)
                {
                    case '.':
                        return ReadFraction();

                    case 'e':
                    case 'E':
                        return ReadExponent();

                    case 'j':
                    case 'J':
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(LiteralParser.ParseImaginary(GetTokenString()));

                    case 'l':
                    case 'L':
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(LiteralParser.ParseBigInteger(GetTokenString(), b));

                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        break;

                    default:
                        BufferBack();
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(ParseInteger(GetTokenString(), b));
                }
            }
        }

        private Token ReadBinaryNumber()
        {
            int bits = 0;
            int iVal = 0;
            bool useBigInt = false;
            BigInteger bigInt = BigInteger.Zero;
            while (true)
            {
                int ch = NextChar();
                switch (ch)
                {
                    case '0':
                        if (iVal != 0)
                        {
                            // ignore leading 0's...
                            goto case '1';
                        }
                        break;
                    case '1':
                        bits++;
                        if (bits == 32)
                        {
                            useBigInt = true;
                            bigInt = (BigInteger)iVal;
                        }

                        if (bits >= 32)
                        {
                            bigInt = (bigInt << 1) | (ch - '0');
                        }
                        else
                        {
                            iVal = iVal << 1 | (ch - '0');
                        }
                        break;
                    case 'l':
                    case 'L':
                        MarkTokenEnd();

                        return new ConstantValueToken(useBigInt ? bigInt : (BigInteger)iVal);
                    default:
                        BufferBack();
                        MarkTokenEnd();

                        return new ConstantValueToken(useBigInt ? (object)bigInt : (object)iVal);
                }
            }
        }

        private Token ReadOctalNumber()
        {
            while (true)
            {
                int ch = NextChar();

                switch (ch)
                {
                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                        break;

                    case 'l':
                    case 'L':
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(LiteralParser.ParseBigInteger(GetTokenSubstring(2, TokenLength - 2), 8));

                    default:
                        BufferBack();
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(ParseInteger(GetTokenSubstring(2), 8));
                }
            }
        }

        private Token ReadHexNumber()
        {
            while (true)
            {
                int ch = NextChar();

                switch (ch)
                {
                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                    case 'a':
                    case 'b':
                    case 'c':
                    case 'd':
                    case 'e':
                    case 'f':
                    case 'A':
                    case 'B':
                    case 'C':
                    case 'D':
                    case 'E':
                    case 'F':
                        break;

                    case 'l':
                    case 'L':
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(LiteralParser.ParseBigInteger(GetTokenSubstring(2, TokenLength - 3), 16));

                    default:
                        BufferBack();
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(ParseInteger(GetTokenSubstring(2), 16));
                }
            }
        }

        private Token ReadFraction()
        {
            while (true)
            {
                int ch = NextChar();

                switch (ch)
                {
                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        break;

                    case 'e':
                    case 'E':
                        return ReadExponent();

                    case 'j':
                    case 'J':
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(LiteralParser.ParseImaginary(GetTokenString()));

                    default:
                        BufferBack();
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(ParseFloat(GetTokenString()));
                }
            }
        }

        private Token ReadExponent()
        {
            int ch = NextChar();

            if (ch == '-' || ch == '+')
            {
                ch = NextChar();
            }

            while (true)
            {
                switch (ch)
                {
                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                        ch = NextChar();
                        break;

                    case 'j':
                    case 'J':
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(LiteralParser.ParseImaginary(GetTokenString()));

                    default:
                        BufferBack();
                        MarkTokenEnd();

                        // TODO: parse in place
                        return new ConstantValueToken(ParseFloat(GetTokenString()));
                }
            }
        }

        #endregion

        private Token ReadName()
        {
            BufferBack();
            int ch = NextChar();

            while (IsNamePart(ch))
            {
                ch = NextChar();
            }
            BufferBack();

            Token t;
            MarkTokenEnd();
            if (Tokens.AllKeywords.TryGetValue(GetTokenString(), out t)) return t;

            NameToken token;
            if (!_names.TryGetValue(_currentName, out token))
            {
                string name = GetTokenString();
                token = _names[name] = new TermNameToken(name, TokenKind.Name);
            }

            return token;
        }


        /*
        private const char LF = '\u000A';
        private const char FF = '\u000C';
        private const char CR = '\u000D';
        private const char SU = '\u001A';
        private char _ch;
        private int _charOffset = 0;
        private int _lineStartOffset = 0;
        private int _lastLineStartOffset = 0;
        private int _lastUnicodeOffset = -1;
        private Token _next, _prev,_token;
        private Names.TermName _name;

        private bool isDigit(char c) { return char.IsDigit(c); }
        private int _openComments = 0;
        private void putCommentChar()
        {
            nextChar();
        }

        private void skipLineComment()
        {
            switch(_ch)
            {
                case SU:
                case CR:
                case LF:
                    break;
                default:
                    nextChar();
                    skipLineComment();
            }
        }
        private void maybeOpen()
        {
            putCommentChar();
            if(_ch == '#')
            {
                putCommentChar();
                _openComments += 1;
            }
        }
        private bool maybeClose()
        {
            putCommentChar();
            if(_ch == '/')
            {
                putCommentChar();
                _openComments -= 1;
                if (_openComments == 0) return true;
            }
            return false;
        }
        public void skipNestedComments()
        {
            switch(_ch)
            {
                case '/': maybeOpen(); skipNestedComments(); break;
                case '*': if (!maybeClose()) skipNestedComments(); break;
                case SU: throw new Exception("Incomplete Error");
                default: putCommentChar(); skipNestedComments(); break;
            }
        }

        public void skipDocComments() { skipNestedComments(); }
        public void skipBlockComments() { skipNestedComments(); }

        private void skipToCommentEnd(bool isLineComment)
        {
            nextChar();
            if (isLineComment) skipLineComment();
            else
            {
                _openComments = 1;
                var isDocComment = (_ch == '*');
                nextChar();
                if (isDocComment)
                {
                    if (_ch == '/') nextChar();
                    else skipDocComments();
                }
                else skipBlockComments();
            }
        }

        private bool skipComment()
        {
            switch(_ch)
            {
                case '/':
                case '*':
                    skipToCommentEnd((_ch == '/'));
                    return true;
                default: return false;
            }
        }
        private object flushDoc() { return null; }
        private void discardDocBuffer() { }

        private bool isAtEnd() { return _charOffset >= _buffer.Length; }

        private void resume(Token lastCode)
        {
            if(_next.Kind != TokenKind.EMPTY  && add no errors code here )
            {
                throw new Exception("unepected end of input: possible missing '}' in XML Block");
            }
            nextToken();
        }

        public StringBuilder cbuf = new StringBuilder();

        protected void putChar(char c)
        {
            cbuf.Append(c);
        }

        protected bool emitIdentifierDeprecationWarnings() { return true; }

        private void finishedNamed(Token idToken)
        {
            _name = Names.newTermName(cbuf.ToString());
            cbuf.Clear();
            _token = idToken;
            if(idToken.Kind == TokenKind.IDENTIFIER)
            {
                var idx = _name._start - kwOffset;
                if(idx>= 0 && idx<kwArray.Length)
                {
                    _token = kwArray[idx];
                    if(_token.Kind == TokenKind.IDENTIFIER && allowIdent != _name)
                    {
                        if (_name == nme.MACROkw) throw new Exception("Name is now a reserved word; usage as an identifier is disallowed");
                        else if (emitIdentifierDeprecationWarnings) throw new Exception("deprecation");
                    }
                }
            }
        }

        private string _strVal;
        private void setStrVal()
        {
            _strVal = cbuf.ToString();
            cbuf.Clear();
        }

        private List<Token> sepRegions = new List<Token>();

        private bool inStringInterpolation() { return (sepRegions.Count != 0 && sepRegions[0].Kind == TokenKind.STRINGLIT);}
        private bool inMultiLineInterpolation()
        {
            var temp = new List<Token>();
            temp = sepRegions;
            temp.RemoveAt(0);
            return inStringInterpolation() && temp.Count != 0 && temp[0].Kind == TokenKind.STRINGLIT;
        }

        public int skipToken()
        {
            var off = _Offset;
            nextToken();
            return off;
        }

        private Names.Name _allowIdent;

        private void nextTokenAllow(Names.Name name)
        {
            var prev = _allowIdent;
            _allowIdent = name;
            try
            {
                nextToken();
            }
            finally
            {
                _allowIdent = prev;
            }
        }

        internal void nextToken()
        {
            var lastToken = _token;
            switch(lastToken.Kind)
            {
                case TokenKind.LPAREN: sepRegions = sepRegions.Insert(0,)
            }
        }

*/
        class TokenEqualityComparer : IEqualityComparer<object>
        {
            private readonly Tokenizer _tokenizer;

            public TokenEqualityComparer(Tokenizer tokenizer)
            {
                _tokenizer = tokenizer;
            }

            #region IEqualityComparer<object> Members

            bool IEqualityComparer<object>.Equals(object x, object y)
            {
                if (x == _currentName)
                {
                    if (y == _currentName)
                    {
                        return true;
                    }
                    return Equals((string)y);
                }
                else if (y == _currentName)
                {
                    return Equals((string)x);
                }
                else
                {
                    return (string)x == (string)y;
                }
            }

            public int GetHashCode(object obj)
            {
                int result = 5381;
                if (obj == _currentName)
                {
                    char[] buffer = _tokenizer._buffer;
                    int start = _tokenizer._start, end = _tokenizer._tokenEnd;
                    for (int i = start; i < end; i++)
                    {
                        int c = buffer[i];
                        result = unchecked(((result << 5) + result) ^ c);
                    }
                }
                else
                {
                    string str = (string)obj;
                    for (int i = 0; i < str.Length; i++)
                    {
                        int c = str[i];
                        result = unchecked(((result << 5) + result) ^ c);
                    }
                }
                return result;
            }

            public bool Equals(string value)
            {
                int len = _tokenizer._tokenEnd - _tokenizer._start;
                if (len != value.Length)
                {
                    return false;
                }
                var buffer = _tokenizer._buffer;
                for (int i = 0, bufferIndex = _tokenizer._start; i < value.Length; i++, bufferIndex++)
                {
                    if (value[i] != buffer[bufferIndex])
                    {
                        return false;
                    }
                }
                return true;
            }

            #endregion
        }

        public int GroupingLevel
        {
            get { return _state.ParenLevel + _state.BraceLevel + _state.BracketLevel; }
        }

        public bool EndContinues
        {
            get { return _endContinues; }
        }

        private bool ReadNewLine()
        {
            return false;
        }

        private object ParseInteger(string s, int radix)
        {
            return Convert.ToInt32(s);
        }

        private object ParseFloat(string s)
        {
            return null;
        }

        internal static bool TryGetEncoding(Encoding defaultEncoding, string line, ref Encoding enc, out string encName)
        {
            encName = "";
            return false;
        }

        private void ReportSyntaxError(SourceSpan span, string message, int errorCode)
        {
            _errors.Add(_sourceUnit, message, span, errorCode, Severity.FatalError);
        }
        private void ReportSyntaxError(IndexSpan span, string message, int errorCode)
        {
            _errors.Add(_sourceUnit, message, new SourceSpan(IndexToLocation(span.Start), IndexToLocation(span.End)), errorCode, Severity.FatalError);
        }

        [Conditional("DUMP_TOKENS")]
        private void DumpBeginningOfUnit()
        {
            Console.WriteLine("--- Source Unit: '{0}' ---", _sourceUnit);
        }

        [Conditional("DUMP_TOKENS")]
        private static void DumpToken(Token token)
        {
            Console.WriteLine("{0} `{1}`", token.Kind, token.Image.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\t", "\\t"));
        }

        public int[] GetLineLocations()
        {
            return _newLineLocations.ToArray();
        }
        [Serializable]
        class IncompleteString : IEquatable<IncompleteString>
        {
            public readonly bool IsRaw, IsUnicode, IsTripleQuoted, IsSingleTickQuote;
            public IncompleteString(bool isSingleTickQuote, bool isRaw, bool isUnicode, bool isTriple)
            {
                IsRaw = isRaw;
                IsUnicode = isUnicode;
                IsTripleQuoted = isTriple;
                IsSingleTickQuote = isSingleTickQuote;
            }

            public override bool Equals(object obj)
            {
                IncompleteString oth = obj as IncompleteString;
                if (oth != null)
                {
                    return Equals(oth);
                }
                return false;
            }

            public override int GetHashCode()
            {
                return (IsRaw ? 0x01 : 0) |
                    (IsUnicode ? 0x02 : 0) |
                    (IsTripleQuoted ? 0x04 : 0) |
                    (IsSingleTickQuote ? 0x08 : 0);
            }

            public static bool operator ==(IncompleteString left, IncompleteString right)
            {
                if ((object)left == null) return (object)right == null;
                return left.Equals(right);
            }

            public static bool operator !=(IncompleteString left, IncompleteString right)
            {
                return !(left == right);
            }

            #region IEquatable<String> Members

            public bool Equals(IncompleteString other)
            {
                if (other == null)
                {
                    return false;
                }

                return IsRaw == other.IsRaw &&
                    IsUnicode == other.IsUnicode &&
                    IsTripleQuoted == other.IsTripleQuoted &&
                    IsSingleTickQuote == other.IsSingleTickQuote;
            }

            #endregion
        }

        [Serializable]
        struct State : IEquatable<State>
        {
            public bool LastNewLine;
            public IncompleteString IncompleteString;


            public int ParenLevel, BraceLevel, BracketLevel;

            public State(State state)
            {
                LastNewLine = state.LastNewLine;
                BracketLevel = state.BracketLevel;
                ParenLevel = state.ParenLevel;
                BraceLevel = state.BraceLevel;
                IncompleteString = state.IncompleteString;
            }

            public State(object dummy)
            {
                LastNewLine = false;
                BracketLevel = ParenLevel = BraceLevel = 0;
                IncompleteString = null;
            }

            public override bool Equals(object obj)
            {
                if (obj is State)
                {
                    State other = (State)obj;
                    return other == this;
                }
                else
                {
                    return false;
                }
            }

            public override int GetHashCode()
            {
                return base.GetHashCode();
            }

            public static bool operator ==(State left, State right)
            {
                if (left == null) return right == null;

                return left.BracketLevel == right.BracketLevel &&
                       left.BraceLevel == right.BraceLevel &&
                       left.ParenLevel == right.ParenLevel &&
                       left.LastNewLine == right.LastNewLine &&
                       left.IncompleteString == right.IncompleteString;
            }

            public static bool operator !=(State left, State right)
            {
                return !(left == right);
            }

            #region IEquatable<State> Members

            public bool Equals(State other)
            {
                return this.Equals(other);
            }

            #endregion
        }

        #region Buffer Access

        private string GetTokenSubstring(int offset)
        {
            return GetTokenSubstring(offset, _tokenEnd - _start - offset);
        }

        private string GetTokenSubstring(int offset, int length)
        {
            Debug.Assert(_tokenEnd != -1, "Token end not marked");
            Debug.Assert(offset >= 0 && offset <= _tokenEnd - _start && length >= 0 && length <= _tokenEnd - _start - offset);

            return new String(_buffer, _start + offset, length);
        }

        [Conditional("DEBUG")]
        private void CheckInvariants()
        {
            Debug.Assert(_buffer.Length >= 1);
            Debug.Assert(_start >= 0 && _start <= _end);
            Debug.Assert(_end >= 0 && _end <= _buffer.Length);
            Debug.Assert(_position >= _start);
            Debug.Assert(_tokenEnd >= -1 && _tokenEnd <= _end);
        }

        private int Peek()
        {
            if (_position >= _end)
            {
                RefillBuffer();
                if (_position >= _end)
                {
                    return EOF;
                }
            }
            Debug.Assert(_position < _end);
            return _buffer[_position];
        }

        private int ReadLine()
        {
            int ch;
            do { ch = NextChar(); } while (ch != EOF && !IsEoln(ch));
            BufferBack();
            return ch;
        }

        private void MarkTokenEnd()
        {
            CheckInvariants();
            _tokenEnd = System.Math.Min(_position, _end);
            int token_length = _tokenEnd - _start;
            _tokenEndIndex = _tokenStartIndex + token_length;

            DumpToken();
            CheckInvariants();
        }

        [Conditional("DUMP_TOKENS")]
        private void DumpToken()
        {
            Console.WriteLine("--> `{0}` {1}", GetTokenString().Replace("\r", "\\r").Replace("\n", "\\n").Replace("\t", "\\t"), TokenSpan);
        }

        private void BufferBack()
        {
            SeekRelative(-1);
        }

        internal string GetTokenString()
        {
            return new String(_buffer, _start, _tokenEnd - _start);
        }

        private int TokenLength
        {
            get { return _tokenEnd - _start; }
        }

        private void SeekRelative(int disp)
        {
            CheckInvariants();
            Debug.Assert(disp >= _start - _position);
            _position += disp;
            CheckInvariants();
        }

        private SourceLocation BufferTokenEnd
        {
            get { return IndexToLocation(_tokenEndIndex); }
        }

        private IndexSpan BufferTokenSpan
        {
            get { return new IndexSpan(_tokenStartIndex, _tokenEndIndex - _tokenStartIndex); }
        }

        private bool NextChar(int ch)
        {
            CheckInvariants();
            if (Peek() == ch)
            {
                _position++;
                CheckInvariants();
                return true;
            }
            else
            {
                return false;
            }
        }

        private int NextChar()
        {
            int result = Peek();
            _position++;
            return result;
        }

        private bool AtBeginning
        {
            get { return _position == 0 && !_bufferResized; }
        }

        private int CurrentIndex
        {
            get { return _tokenStartIndex + Math.Min(_position, _end) - _start; }
        }

        private void DiscardToken()
        {
            CheckInvariants();
            if (_tokenEnd == -1) MarkTokenEnd();

            _start = _tokenEnd;
            _tokenStartIndex = _tokenEndIndex;
            _tokenEnd = -1;
#if DEBUG
            _tokenEndIndex = -1;
#endif
            CheckInvariants();
        }

        private int ReadEolnOpt(int current)
        {
            if (current == '\n') return 1;

            if (current == '\r' && _multiEolns)
            {
                if (Peek() == '\n')
                {
                    SeekRelative(+1);
                    return 2;
                }
                return 1;
            }
            return 0;
        }

        private bool IsEoln(int current)
        {
            if (current == '\n') return true;
            if (current == '\r' && _multiEolns)
            {
                if (Peek() == '\n') return true;
                return true;
            }
            return false;
        }

        private void RefillBuffer()
        {
            if (_end == _buffer.Length)
            {
                int new_size = System.Math.Max(System.Math.Max((_end - _start) * 2, _buffer.Length), _position);
                ResizeInternal(ref _buffer, new_size, _start, _end - _start);
                _end -= _start;
                _position -= _start;
                _start = 0;
                _bufferResized = true;
            }
            int count = _reader.Read(_buffer, _end, _buffer.Length - _end);
            _end += count;
            ClearInvalidChars();
        }

        private static void ResizeInternal(ref char[] array, int newSize, int start, int count)
        {
            Debug.Assert(array != null && newSize > 0 && count >= 0 && newSize >= count && start >= 0);
            char[] result = (newSize != array.Length) ? new char[newSize] : array;
            Buffer.BlockCopy(array, start * sizeof(char), result, 0, count * sizeof(char));
            array = result;
        }

        [Conditional("DEBUG")]
        private void ClearInvalidChars()
        {
            for (int i = 0; i < _start; i++) _buffer[i] = '\0';
            for (int i = _end; i < _buffer.Length; i++) _buffer[i] = '\0';
        }

        #endregion
    }
}
